{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation for the cold-start setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "log/cold-start/0417_2207_HDN-DDI_fold2.log [0.7894, 0.8765, 0.8735, 0.7739, 0.8932, 0.9577, 0.956, 0.8944]\n",
      "log/cold-start/0419_1142_HDN-DDI_fold0.log [0.8113, 0.8963, 0.8941, 0.7979, 0.8962, 0.9594, 0.959, 0.8964]\n",
      "log/cold-start/0419_1237_HDN-DDI_fold1.log [0.8004, 0.8763, 0.8732, 0.7981, 0.88, 0.9518, 0.9501, 0.884]\n",
      "log/cold-start/0419_1513_HDN-DDI_fold2.log [0.8085, 0.8984, 0.9032, 0.7881, 0.9108, 0.9703, 0.9698, 0.9095]\n",
      "log/cold-start/0419_1755_HDN-DDI_fold0.log [0.8121, 0.9011, 0.9089, 0.7889, 0.8951, 0.9578, 0.9603, 0.8942]\n",
      "log/cold-start/0419_1903_HDN-DDI_fold1.log [0.7807, 0.8732, 0.8762, 0.755, 0.8812, 0.9499, 0.9524, 0.8794]\n",
      "log/cold-start/0419_2218_HDN-DDI_fold2.log [0.7707, 0.863, 0.8592, 0.7506, 0.8937, 0.9583, 0.9583, 0.8945]\n",
      "log/cold-start/0420_0115_HDN-DDI_fold0.log [0.8464, 0.9306, 0.9314, 0.8362, 0.9147, 0.9708, 0.9712, 0.9156]\n",
      "log/cold-start/0420_0225_HDN-DDI_fold1.log [0.7923, 0.8733, 0.8747, 0.778, 0.8926, 0.9566, 0.9573, 0.8909]\n",
      "HDN-DDI total: 9\n",
      "[0.80131111 0.88763333 0.88826667 0.78518889 0.89527778 0.95917778\n",
      " 0.95937778 0.89543333]\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "\n",
    "def parse_lot_best_result(log_file):\n",
    "    scores = []\n",
    "\n",
    "    s1_pattern = '.*s1_acc:(.*?),.*s1_roc:(.*?),.*s1_f1:(.*?),.*s1_ap:(.*)'\n",
    "    s2_pattern = '.*s2_acc:(.*?),.*s2_roc:(.*?),.*s2_f1:(.*?),.*s2_ap:(.*)'\n",
    "    with open(log_file) as f:\n",
    "        lines = f.readlines()\n",
    "        s1_match_res = re.match(s1_pattern, lines[-2].strip())\n",
    "        s2_match_res = re.match(s2_pattern, lines[-1].strip())\n",
    "        if s1_match_res:\n",
    "            for i in range(0,4):\n",
    "                scores.append(float(s1_match_res.group(i+1).strip()))\n",
    "            scores[2], scores[3] = scores[3], scores[2]\n",
    "        if s2_match_res:\n",
    "            for i in range(0,4):\n",
    "                scores.append(float(s2_match_res.group(i+1).strip()))\n",
    "            scores[-1], scores[-2] = scores[-2], scores[-1]\n",
    "    return scores\n",
    "\n",
    "keyword = 'HDN-DDI' \n",
    "cond = lambda x: keyword in x\n",
    "dir = f'log/cold-start'\n",
    "files = [os.path.join(dir, f) for f in os.listdir(dir) if cond(f)]\n",
    "files = sorted(files)\n",
    "scores = []\n",
    "for f in files:\n",
    "    s = parse_lot_best_result(f)\n",
    "    if len(s)>0:\n",
    "        print(f, s)\n",
    "        scores.append(s)\n",
    "print(keyword, 'total:', len(files))\n",
    "print(np.mean(scores, axis=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation for the warm-start setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "log/warm-start/0313_1623_HDNDDI_warm-start_fold0.log [0.9732, 0.9963, 0.9959, 0.9736]\n",
      "log/warm-start/0313_1822_HDNDDI_warm-start_fold1.log [0.9845, 0.9983, 0.9981, 0.9845]\n",
      "log/warm-start/0313_2123_HDNDDI_warm-start_fold2.log [0.9754, 0.9967, 0.9964, 0.9756]\n",
      "log/warm-start/0314_2248_HDNDDI_warm-start_fold0.log [0.9809, 0.9979, 0.9975, 0.9811]\n",
      "log/warm-start/0315_0126_HDNDDI_warm-start_fold1.log [0.9806, 0.9981, 0.9979, 0.9808]\n",
      "log/warm-start/0315_0408_HDNDDI_warm-start_fold2.log [0.9847, 0.9986, 0.9986, 0.9848]\n",
      "log/warm-start/0315_0706_HDNDDI_warm-start_fold0.log [0.9854, 0.9987, 0.9986, 0.9854]\n",
      "log/warm-start/0315_1012_HDNDDI_warm-start_fold1.log [0.9796, 0.9979, 0.9977, 0.9798]\n",
      "log/warm-start/0315_1302_HDNDDI_warm-start_fold2.log [0.9843, 0.9986, 0.9985, 0.9844]\n",
      "warm-start total: 9\n",
      "[0.98095556 0.9979     0.99768889 0.98111111]\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "\n",
    "def parse_lot_best_result(log_file):\n",
    "    scores = []\n",
    "\n",
    "    pattern = '.*test_acc:(.*?),.*test_roc:(.*?),.*test_f1:(.*?),.*test_ap:(.*)'\n",
    "    with open(log_file) as f:\n",
    "        lines = f.readlines()\n",
    "        match_res = re.match(pattern, lines[-1].strip())\n",
    "        if match_res:\n",
    "            for i in range(0,4):\n",
    "                scores.append(float(match_res.group(i+1).strip()))\n",
    "            scores[2], scores[3] = scores[3], scores[2]\n",
    "    return scores\n",
    "\n",
    "keyword = 'warm-start' \n",
    "cond = lambda x: keyword in x\n",
    "dir = f'log/warm-start/'\n",
    "files = [os.path.join(dir, f) for f in os.listdir(dir) if cond(f)]\n",
    "files = sorted(files)\n",
    "scores = []\n",
    "for f in files:\n",
    "    s = parse_lot_best_result(f)\n",
    "    if len(s)>0:\n",
    "        print(f, s)\n",
    "        scores.append(s)\n",
    "print(keyword, 'total:', len(files))\n",
    "print(np.mean(scores, axis=0))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch-2.1",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
